#include "SplitToolCppJieba.h"

SplitToolCppJieba::SplitToolCppJieba()
    : _dict_path("dict/jieba.dict.utf8"), _model_path("dict/hmm_model.utf8"), _user_dict_path("dict/user.dict.utf8"), _idf_path("dict/idf.utf8"), _stop_word_path("dict/stop_words.utf8"), _jieba(_dict_path, _model_path, _user_dict_path, _idf_path, _stop_word_path)
{
    // const char* dict_path = "../dict/jieba.dict.utf8";
    // const char* model_path = "../dict/hmm_model.utf8";
    // const char* user_dict_path = "../dict/user.dict.utf8";
    // const char* idf_path = "../dict/idf.utf8";
    // const char* stop_word_path = "../dict/stop_words.utf8r;
}

SplitToolCppJieba::~SplitToolCppJieba()
{
}

unordered_set<string> SplitToolCppJieba::StopWords(const string &filename)
{
    unordered_set<string> stopWords;
    ifstream ifs;
    ifs.open(filename);
    if (!ifs.is_open())
    {
        cerr << "打开文件失败: " << filename << "\n";
    }

    string word;
    while (getline(ifs, word))
    {
        word.erase(0, word.find_first_not_of(" \t\n\r"));
        word.erase(word.find_last_not_of(" \t\r\n") + 1);
        stopWords.insert(word);
    }

    return stopWords;
}

string SplitToolCppJieba::tokenize(const string &text)
{
    string sentence;

    for (char c : text)
    {
        if (c != '\r' && c != '\n' && c != ' ' && c != '\t')
        {
            sentence += c;
        }
    }

    return sentence;
}

vector<string> SplitToolCppJieba::cut(const string &sentence)
{
    static unordered_set<string> cnStopWords = StopWords("conf/stop_words_zh.txt");
    vector<string> AllWords;

    vector<string> words;
    string newSentence = tokenize(sentence);
    _jieba.Cut(newSentence, words, true);

    // 使用过滤的方式减少多余操作
    vector<string> filteredWords;
    for (const auto &word : words)
    {
        if (cnStopWords.find(word) == cnStopWords.end())
        {
            filteredWords.push_back(word);
        }
    }
    AllWords.insert(AllWords.end(), filteredWords.begin(), filteredWords.end());

    return AllWords;
}
